Импорт данных¶
In [3]:
data = pd.read_csv('data/processed/tep_data.csv', index_col='Index')
print(f'Len of dataset: {data.shape[0]}')
Гиперпараметры модели¶
In [7]:
from src.models.autoencoder import build_autoencoder
import keras
create_params = dict(
input_shape=window_length,
hidden_layer_size=16,
hidden_layer_activation=None,
reg_strength=0.001,
input_dropout=-1,
)
compile_params = dict(
optimizer=keras.optimizers.Adam(lr=0.001),
loss='mse',
)
fit_params = dict(
batch_size=64,
epochs=100,
verbose=1,
callbacks=[keras.callbacks.ReduceLROnPlateau(patience=5),
keras.callbacks.EarlyStopping(min_delta=0.01, patience=15)]
)
model_fn = lambda: build_autoencoder(create_params, compile_params)
Обучение модели¶
Кросс-валидация¶
In [20]:
from src.utils import cross_validate
X_comp = rolling_window(X_one_component.to_numpy()[:, None], window_length=window_length).squeeze()
results = cross_validate(model_fn, 3, fit_params, X_comp, X_comp)
In [21]:
from src.visualization.visualize import visualize_cv_result
visualize_cv_result(results, f'Autoencoder loss')
Извлечём слой, который "эмбеддит"¶
In [26]:
import keras
from keras.layers import Input
from keras.models import Model, load_model
model = load_model('models/onedimensional_autoencoder.h5')
inp = Input(shape=(window_length,))
encoder_layer = model.layers[0]
encoded = encoder_layer(inp)
encoder = Model(inputs=inp, outputs=encoded)
Добавим аномалии в нормальные данные¶
In [22]:
from src.data.generate import generate_anomalies
data_with_anom = data[str(component)].copy()
anom_amount = 100
anom_idxs_start = np.random.choice(len(data_with_anom),
anom_amount,
replace=False)
anom_lens = np.zeros(anom_amount, dtype=int)
anom_idxs = np.zeros(len(data_with_anom), dtype=int)
for i, idx in enumerate(anom_idxs_start):
l = np.random.randint(window_length, window_length * 2)
if idx + l > data_with_anom.shape[0]:
raise Exception('Try again, index out of range (FIXME THEN)')
anom_lens[i] = l
anoms = generate_anomalies(l)
anom_idxs[idx:idx + l] = 1
data_with_anom[idx:idx + l] = anoms[np.random.randint(len(anoms))]
Кластеризация эмбеддингов¶
KMeans¶
Найдём оптимальное значение количества кластеров по elbow
In [28]:
from sklearn.cluster import KMeans
from yellowbrick.cluster import KElbowVisualizer
kmeans = KMeans()
visualizer = KElbowVisualizer(kmeans, k=(2,10), metric='distortion')
visualizer.fit(encoded)
visualizer.poof()
Out[28]:
In [29]:
from yellowbrick.cluster import InterclusterDistance
km = KMeans(3)
intra_visualizer = InterclusterDistance(km)
intra_visualizer.fit(encoded) # Fit the data to the visualizer
intra_visualizer.poof() # Draw/show/poof the data
Out[29]:
Будем считать, что 0-ой и 2-ой кластеры полностью аномальные (выбрали наименьшие кластеры)
In [33]:
pred = km.predict(encoded)
maximal_cluster = pd.Series(pred).value_counts().idxmax()
anoms_pred = np.where(pred != maximal_cluster)[0]
In [35]:
from sklearn.metrics import recall_score, precision_score
from src.utils import intersection_over_true
recall = intersection_over_true(data.shape[0], anom_idxs_start, anom_lens, anoms_pred, window_length, recall_score)
precision = intersection_over_true(data.shape[0], anom_idxs_start, anom_lens, anoms_pred, window_length, precision_score)
print(f'Полнота={recall:.3f}, Точность={precision:.3f}')
In [37]:
# Blue - anomalies predicted by model
# Red - anomalies that was not catched by model
# Green - original data, that is neither anomaly nor predicted as anomaly
show(p)
In [ ]: